#!/bin/bash
#####################################################################
# 功能：Tomcat集群管理脚本
# 描述：用于管理Tomcat单机多节点集群，包括：
#      1. 节点状态监控和管理
#      2. 日志实时查看和分析
#      3. 服务启停和重启
#      5. Nginx负载均衡管理
#      6. ActiveMQ服务管理
#      7. 系统状态监控
# 环境要求：
#      - Linux系统
#      - JDK 1.8+
#      - curl工具
# 作者：wurenlong & Copilot Pro
# 版本：1.0.0
#####################################################################

# 错误处理和环境设置
set -e  # 遇到错误立即退出
export LANG="zh_CN.UTF-8"  # 设置中文环境
trap 'echo -e "${RED}错误：程序异常退出，请检查日志${NC}"; exit 1' ERR  # 错误捕获

#####################################################################
# 基础环境配置
#####################################################################

# JDK环境配置
JAVA_HOME="/data/jdk1.8.0_411"
export PATH="$JAVA_HOME/bin:$PATH"
export CLASSPATH=".:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar"

# 基础路径配置
BASE_PATH="/data/wy-zwdt"                         # 应用基础路径
PROJECT_PATH="$BASE_PATH/webapp"                  # 项目部署路径
BACKUP_PATH="$BASE_PATH/backup"                   # 备份存储路径
ACTIVEMQ_PATH="$BASE_PATH/apache-activemq-5.16.2" # ActiveMQ安装路径

# Tomcat相关配置
TOMCAT_NAME_PREFIX="apache-tomcat-8.5.100-"       # Tomcat文件夹名前缀，多个节点会有不同的后缀

# Nginx相关配置
NGINX_PATH="/usr/local/nginx"                     # Nginx安装路径
NGINX_PID="/var/run/nginx.pid"                    # Nginx PID文件
NGINX_CONF="$NGINX_PATH/conf/nginx.conf"          # Nginx主配置文件
NGINX_ERROR_LOG="$NGINX_PATH/logs/error.log"      # 错误日志
NGINX_ACCESS_LOG="$NGINX_PATH/logs/access.log"    # 访问日志
declare -a NGINX_PORTS=("80" "443")               # 监控端口列表

# 应用配置
HEALTH_CHECK_URL="/szdt/"                         # 健康检查URL
TIMEOUT=10                                        # 默认超时时间(秒)

# Tomcat节点配置 - 格式：节点名=端口号
declare -A NODES=(
    ["node1"]="8010"    # 节点1
    ["node2"]="8011"    # 节点2
    ["node3"]="8012"    # 节点3
)

# 终端颜色定义
RED='\033[0;31m'        # 错误/警告
GREEN='\033[0;32m'      # 成功/正常
YELLOW='\033[1;33m'     # 提示/警告
NC='\033[0m'            # 重置颜色

# 分隔线
SEPARATOR="-----------------------------------------------"

# 检查必要条件
check_prerequisites() {
    # 检查JDK
    if [ ! -d "$JAVA_HOME" ]; then
        echo "错误: 未找到JDK路径 $JAVA_HOME"
        echo "请确保JDK已正确安装并设置了正确的JAVA_HOME路径"
        exit 1
    fi
    
    if ! "$JAVA_HOME/bin/java" -version >/dev/null 2>&1; then
        echo "错误: Java命令执行失败，请检查JDK安装"
        exit 1
    fi

    command -v curl >/dev/null 2>&1 || { echo "需要curl但未安装" >&2; exit 1; }
}

# 获取Tomcat节点路径的通用函数
get_tomcat_node_path() {
    local node=$1
    echo "$BASE_PATH/$TOMCAT_NAME_PREFIX$node"
}

# 获取进程ID的通用方法
get_tomcat_node_pid() {
    local node=$1
    local tomcat_path=$(get_tomcat_node_path "$node")
    local pid_file="$tomcat_path/logs/catalina-daemon.pid"
    local pid=""
    
    # 首先检查PID文件
    if [ -f "$pid_file" ]; then
        pid=$(cat "$pid_file")
        # 验证PID是否有效 - 只取第一个PID
        pid=$(echo "$pid" | awk '{print $1}')
        if ps -p $pid > /dev/null 2>&1; then
            echo $pid
            return 0
        else
            # PID文件存在但进程不存在，清理无效的PID文件
            rm -f "$pid_file"
        fi
    fi
    
    # 如果PID文件不存在或无效，尝试通过进程名查找
    # 确保只获取一个PID
    pid=$(pgrep -f "$TOMCAT_NAME_PREFIX$node" | head -n 1)
    if [ -n "$pid" ]; then
        # 找到进程，写入PID文件以便后续使用
        echo $pid > "$pid_file"
        echo $pid
        return 0
    fi
    
    # 都没找到，返回空
    echo ""
    return 1
}


# 获取Nginx进程ID的通用方法
get_nginx_pid() {
    local pid=""
    
    # 首先检查PID文件
    if [ -f "$NGINX_PID" ]; then
        pid=$(cat "$NGINX_PID")
        # 验证PID是否有效
        if ps -p $pid > /dev/null 2>&1; then
            echo $pid
            return 0
        else
            # PID文件存在但进程不存在，清理无效的PID文件
            rm -f "$NGINX_PID"
        fi
    fi
    
    # 如果PID文件不存在或无效，尝试通过进程名查找主进程
    pid=$(pgrep -f "nginx: master process" | head -n 1)
    if [ -n "$pid" ]; then
        # 找到进程，写入PID文件以便后续使用
        echo $pid > "$NGINX_PID"
        echo $pid
        return 0
    fi
    
    # 都没找到，返回空
    echo ""
    return 1
}

# 检查HTTP状态的通用方法
check_tomcat_node_http_status() {
    local port=$1
    local url=${2:-$HEALTH_CHECK_URL}  # 如果未指定URL则使用默认健康检查URL
    local timeout=${3:-$TIMEOUT}        # 如果未指定超时则使用默认超时时间
    
    if timeout $timeout curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:${port}${url}" | grep -q "200"; then
        return 0  # 返回成功
    else
        return 1  # 返回失败
    fi
}

# 获取节点状态和PID
get_tomcat_node_status() {
    local node=$1
    local port=${NODES[$node]}
    local tomcat_path=$(get_tomcat_node_path "$node")
    local pid=$(get_tomcat_node_pid $node)
    local status="停止"
    local http_status="未响应"
    local memory_usage=""
    local cpu_usage=""

    if [ -n "$pid" ]; then
        status="${GREEN}运行中${NC}"
        
        # 查找所有相关Tomcat进程 - 用于计算总内存和CPU使用
        local all_pids=$(pgrep -f "$TOMCAT_NAME_PREFIX$node")
        
        # 计算总内存使用
        local total_mem=0
        for p in $all_pids; do
            local mem=$(ps -o rss= -p $p 2>/dev/null)
            if [[ -n "$mem" && "$mem" =~ ^[0-9]+$ ]]; then
                total_mem=$((total_mem + mem))
            fi
        done
        memory_usage=$(awk "BEGIN {printf \"%.2f GB\", $total_mem/1024/1024}")
        
        # 计算总CPU使用
        local total_cpu=0
        for p in $all_pids; do
            local cpu=$(ps -p $p -o %cpu= 2>/dev/null | awk '{print $1}')
            if [[ -n "$cpu" && "$cpu" =~ ^[0-9.]+$ ]]; then
                total_cpu=$(echo "$total_cpu + $cpu" | bc)
            fi
        done
        cpu_usage=$(awk "BEGIN {printf \"%.1f%%\", $total_cpu}")
        
        if check_tomcat_node_http_status ${port}; then
            http_status="${GREEN}正常${NC}"
        else
            http_status="${RED}异常${NC}"
        fi
    else
        pid="${RED}无${NC}"
    fi

    echo -e "[$node] - 端口: $port, 状态: $status, PID: $pid, HTTP状态: $http_status, CPU: $cpu_usage, 内存: $memory_usage"
}

# 日志管理
view_tomcat_node_logs() {
    local node=$1
    local tomcat_path=$(get_tomcat_node_path "$node")
    local log_file="$tomcat_path/logs/catalina-daemon.out"
    
    clear
    echo "===== $node 日志查看 ====="
    echo "1: 实时查看最新日志"
    echo "2: 查看错误日志"
    echo "b: 返回"
    
    read -e -p "请选择: " log_choice
    case $log_choice in
        1)
            echo "查看实时日志 (Ctrl+C 退出)..."
            tail -f "$log_file"
            ;;
        2)
            echo "查看错误日志 (按q退出)..."
            grep -i "error\|exception" "$log_file" | less
            ;;
    esac
}

# 重启节点
restart_tomcat_node() {
    local node=$1
    local tomcat_path=$(get_tomcat_node_path "$node")
    local startup_timeout=90  # 启动超时时间

    stop_tomcat_node $node
    
    # 启动前检查端口
    if netstat -tln | grep -q ":${NODES[$node]} "; then
        echo -e "${RED}错误: 端口 ${NODES[$node]} 已被占用，请检查${NC}"
        return 1
    fi
    
    echo -n "正在启动 $node..."
    # 将启动命令放入后台执行，并重定向输出
    nohup $tomcat_path/bin/daemon.sh start > "$tomcat_path/logs/startup.log" 2>&1 &
    
    # 等待启动完成
    local start_time=$SECONDS
    local timeout_time=$((SECONDS + startup_timeout))
    local started=false
    
    while [ $SECONDS -lt $timeout_time ]; do
        echo -n "."
        
        # 检查PID文件和进程
        if [ -f "$tomcat_path/logs/catalina-daemon.pid" ]; then
            local new_pid=$(cat "$tomcat_path/logs/catalina-daemon.pid")
            if ps -p $new_pid > /dev/null 2>&1; then
                if netstat -tln | grep -q ":${NODES[$node]} "; then
                    # 显示启动日志中的错误信息
                    if grep -i "error\|exception" "$tomcat_path/logs/startup.log" >/dev/null; then
                        echo -e "\n${YELLOW}警告: 启动过程中发现错误:${NC}"
                        grep -i "error\|exception" "$tomcat_path/logs/startup.log" | tail -n 3
                    fi
                    
                    # 验证服务是否响应
                    if check_tomcat_node_http_status ${NODES[$node]}; then
                        echo -e "\n${GREEN}服务已成功启动 (用时: $((SECONDS - start_time))秒)${NC}"
                        started=true
                        break
                    fi
                fi
            fi
        fi
        
        sleep 1
    done
    
    if ! $started; then
        echo -e "\n${RED}错误: 服务启动失败或响应超时${NC}"
        echo "诊断信息:"
        echo "1. PID文件: $([ -f "$tomcat_path/logs/catalina-daemon.pid" ] && echo "存在" || echo "不存在")"
        echo "2. 进程状态: $(ps aux | grep "$TOMCAT_NAME_PREFIX$node" | grep -v grep || echo "未运行")"
        echo "3. 端口状态: $(netstat -tln | grep "${NODES[$node]}" || echo "未监听")"
        echo "4. 最近日志:"
        tail -n 5 "$tomcat_path/logs/catalina-daemon.out"
        return 1
    fi
    
    get_tomcat_node_status $node
    return 0
}

# 关闭节点
stop_tomcat_node() {
    local node=$1
    local tomcat_path=$(get_tomcat_node_path "$node")
    local pid=$(get_tomcat_node_pid $node)
    
    if [ -n "$pid" ]; then
        echo -n "正在停止 $node..."
        # 将停止命令放入后台执行，并重定向输出
        nohup $tomcat_path/bin/daemon.sh stop > "$tomcat_path/logs/startup.log" 2>&1 &
        
        # 等待进程停止,最多等待60秒
        local count=0
        while [ $count -lt 60 ]; do
            if ! ps -p $pid > /dev/null 2>&1; then
                echo -e "\n${GREEN}服务已停止${NC}"
                break
            fi
            echo -n "."
            sleep 1
            count=$((count + 1))
        done

        if [ $count -eq 60 ]; then
            echo -e "\n${RED}警告: 服务停止超时,尝试强制终止进程...${NC}"
            # 终止所有相关进程，不仅仅是主进程
            local all_pids=$(pgrep -f "$TOMCAT_NAME_PREFIX$node")
            if [ -n "$all_pids" ]; then
                for p in $all_pids; do
                    kill -9 $p 2>/dev/null
                done
                echo -e "${GREEN}已强制终止所有相关进程${NC}"
            else
                kill -9 $pid 2>/dev/null
                echo -e "${GREEN}已强制终止主进程${NC}"
            fi
            sleep 1
        fi
    else 
        echo -e "\n${GREEN}服务已停止${NC}"
    fi
}

# 节点操作菜单
tomcat_node_manager_menu() {
    local node="node$1"
    while true; do
        clear
        echo "====== $node 管理面板 ======"
        get_tomcat_node_status $node
        echo $SEPARATOR
        echo "1: 重启/停止节点"
        echo "2: 查看日志"
        echo "3: 清理Tomcat缓存"
        echo "4: 查看线程状态"
        echo "5: 查看内存使用"
        echo "6: JVM信息"
        echo "7: 监控系统状态"
        echo "b: 返回主菜单"
        echo $SEPARATOR
        
        read -e -p "请输入选项: " choice
        case $choice in
            1)
                read -p "确认要重启 $node 吗？(y重启| s关闭 | n取消): " confirm
                if [ "$confirm" = "y" ]; then
                    restart_tomcat_node $node
                    read -p "按回车继续..."
                fi
                if [ "$confirm" = "s" ]; then
                    local pid=$(get_tomcat_node_pid $node)
                    if [ -n "$pid" ]; then
                        stop_tomcat_node $node
                        read -p "按回车继续..."
                    else
                        echo "节点未运行"
                        sleep 2
                    fi
                fi
                ;;
            2)
                view_tomcat_node_logs $node
                ;;
            3)
                clean_tomcat_node_cache $node
                ;;
            4)
                local pid=$(get_tomcat_node_pid $node)
                if [ -n "$pid" ]; then
                    echo "查看线程状态..."
                    ps -T -p $pid | head -n 20
                    read -p "按回车继续..."
                else
                    echo "节点未运行"
                    sleep 2
                fi
                ;;
            5)
                local pid=$(get_tomcat_node_pid $node)
                if [ -n "$pid" ]; then
                    echo "查看内存使用..."
                    top -b -n 1 -p $pid
                    read -p "按回车继续..."
                else
                    echo "节点未运行"
                    sleep 2
                fi
                ;;
            6)
                local pid=$(get_tomcat_node_pid $node)
                if [ -n "$pid" ]; then
                    echo "查看JVM信息..."
                    jps -v | grep $pid
                    read -p "按回车继续..."
                else
                    echo "节点未运行"
                    sleep 2
                fi
                ;;
            7)
                local pid=$(get_tomcat_node_pid $node)
                if [ -n "$pid" ]; then
                    echo "监控系统状态..."
                    vmstat 1 10
                    read -p "按回车继续..."
                else
                    echo "节点未运行"
                    sleep 2
                fi
                ;;
            b|B)
                break
                ;;
            *)
                echo "无效选项"
                sleep 1
                ;;
        esac
    done
}

# 清理Tomcat缓存
clean_tomcat_node_cache() {
    local node=$1
    local tomcat_path=$(get_tomcat_node_path "$node")
    
    if [ -n "$pid" ]; then
        echo -e "${RED}警告: 节点正在运行!${NC}"
        echo -e "${YELLOW}在运行状态下清理缓存可能导致:${NC}"
        echo "1. 应用程序出现异常"
        echo "2. Session数据丢失"
        echo "3. JSP编译文件丢失需要重新编译"
        echo "4. 临时文件访问冲突"
        echo -e "\n${YELLOW}建议操作:${NC}"
        echo "1. 先停止Tomcat"
        echo "2. 清理缓存"
        echo "3. 重新启动Tomcat"
        echo -e "\n请选择操作:"
        echo "1: 返回上级菜单"
        echo "2: 仍然清理(不建议)"
        echo "3: 先停止再清理"
        
        read -e -p "请输入选项 (1-3): " confirm
        case $confirm in
            1)
                return 0
                ;;
            2)
                echo -e "${RED}警告: 确认强制清理...${NC}"
                sleep 2
                ;;
            3)
                echo -n "正在停止 $node..."
                $tomcat_path/bin/daemon.sh stop
                for i in {1..5}; do
                    echo -n "."
                    sleep 1
                done 
                echo
                ;;
            *)
                echo "已取消清理操作"
                read -p "按回车继续..."
                return 1
                ;;
        esac
    fi

    echo "正在清理Tomcat缓存..."
    local dirs_to_clean=(
        "$tomcat_path/work"
        "$tomcat_path/temp"
    )
    
    local clean_error=0
    for dir in "${dirs_to_clean[@]}"; do
        if [ -d "$dir" ]; then
            echo -n "清理 $dir..."
            if ! rm -rf "${dir:?}"/* 2>/dev/null; then
                echo -e " ${RED}失败${NC}"
                clean_error=1
            else
                mkdir -p "$dir"
                echo -e " ${GREEN}完成${NC}"
            fi
        fi
    done
    
    if [ $clean_error -eq 1 ]; then
        echo -e "\n${YELLOW}提示: 部分文件可能正在被使用，建议在停止Tomcat后再次清理${NC}"
    fi
    
    # 显示清理结果
    echo -e "\n清理完成，当前目录大小:"
    for dir in "${dirs_to_clean[@]}"; do
        if [ -d "$dir" ]; then
            local size=$(du -sh "$dir" 2>/dev/null | cut -f1)
            echo "$(basename "$dir"): $size"
        fi
    done
    
    read -p "按回车继续..."
}

# 获取系统信息
show_system_info() {
    # 标题
    echo "====== Tomcat集群管理系统 ======"
    # 内存信息
    local mem_total=$(free -k | awk 'NR==2{printf "%.1f", $2/1024/1024}')
    local mem_used=$(free -k | awk 'NR==2{printf "%.1f", $3/1024/1024}')
    local mem_free=$(free -k | awk 'NR==2{printf "%.1f", $4/1024/1024}')
    local mem_percent=$(awk "BEGIN {printf \"%.1f\", ($mem_used/$mem_total)*100}")
    echo -e "内存: 总计: ${mem_total}GB | ${GREEN}已用${NC}: ${mem_used}GB (${mem_percent}%) | ${GREEN}空闲${NC}: ${mem_free}GB"
    
    # CPU信息 - 修正计算方法
    local cpu_cores=$(nproc)
    # 使用mpstat或vmstat获取更准确的CPU使用率
    if command -v mpstat >/dev/null 2>&1; then
        # 使用mpstat获取CPU使用信息，计算用户空间+系统空间的百分比
        local cpu_idle=$(mpstat 1 1 | awk '$12 ~ /[0-9.]+/ {print $12}' | tail -1)
        local cpu_usage=$(awk "BEGIN {printf \"%.1f\", 100-$cpu_idle}")
    else
        # 回退方案: 使用top
        local cpu_usage=$(top -bn1 | grep "Cpu(s)" | \
                         sed "s/.*, *\([0-9.]*\)%* id.*/\1/" | \
                         awk '{printf "%.1f", 100-$1}')
    fi
    local cpu_idle=$(awk "BEGIN {printf \"%.1f\", 100-$cpu_usage}")
    
    echo -e "CPU:  核心数: ${cpu_cores} | ${GREEN}使用率${NC}: ${cpu_usage}% | ${GREEN}空闲率${NC}: ${cpu_idle}%"
    
    # 硬盘信息
    local disk_info=$(df -h "$BASE_PATH" | awk 'NR==2{printf "%s\t%s\t%s\t%s", $2, $3, $4, $5}')
    local disk_total=$(echo "$disk_info" | cut -f1)
    local disk_used=$(echo "$disk_info" | cut -f2)
    local disk_free=$(echo "$disk_info" | cut -f3)
    echo -e "硬盘: 总计: ${disk_total} | ${GREEN}已用${NC}: ${disk_used} ($(echo $disk_info | cut -f4)) | ${GREEN}空闲${NC}: ${disk_free}"
    
    echo $SEPARATOR
    df -h | awk 'NR>1 && $1 ~ /^\/dev/ {printf "%-12s: 总计: %-6s | 已用: %-6s | 空闲: %-6s | 使用率: %-6s | 挂载点: %s\n", $1, $2, $3, $4, $5, $6}'
    echo $SEPARATOR

    # 获取ActiveMQ状态
    local mq_status="停止"
    local mq_pid=""
    if [ -f "$ACTIVEMQ_PATH/data/activemq.pid" ]; then
        mq_pid=$(cat "$ACTIVEMQ_PATH/data/activemq.pid")
        if ps -p $mq_pid > /dev/null; then
            mq_status="${GREEN}运行中${NC}"
            local mq_cpu=$(ps -p $mq_pid -o %cpu= | awk '{printf "%.1f%%", $1}')
            local mq_mem=$(ps -o rss= -p $mq_pid | awk '{printf "%.2f GB", $1/1024/1024}')
            echo -e "ActiveMQ - 状态: $mq_status, PID: $mq_pid"
            echo -e "资源使用 - CPU: $mq_cpu, 内存: $mq_mem"
            echo -e "Web控制台: ${GREEN}http://localhost:8161${NC}"
        else
            rm -f "$ACTIVEMQ_PATH/data/activemq.pid"
            mq_pid="${RED}无效${NC}"
        fi
    fi
    echo $SEPARATOR

    # 获取Nginx状态
    local nginx_status="停止"
    local nginx_pid=$(get_nginx_pid)
    if [ -n "$nginx_pid" ]; then
        nginx_status="${GREEN}运行中${NC}"
        local worker_count=$(pgrep -c -P $nginx_pid)
        local nginx_cpu=0
        local nginx_mem=0
        for wpid in $(pgrep -P $nginx_pid); do
            nginx_cpu=$(echo "$nginx_cpu + $(ps -p $wpid -o %cpu= | tr -d ' ')" | bc)
            nginx_mem=$(ps -o rss= -p $wpid | awk '{mem += $1} END {printf "%.2f", mem/1024/1024}')
        done
        echo -e "Nginx - 状态: $nginx_status, PID: $nginx_pid"
        echo -e "Worker进程: $worker_count, 活动连接: ${nginx_connections:-0}"
        echo -e "资源使用 - CPU: ${nginx_cpu}%, 内存: ${nginx_mem}GB"
        echo -n "端口监听: "
        local first=true
        for port in "${NGINX_PORTS[@]}"; do
            if ! $first; then echo -n " | "; fi
            if netstat -tupln 2>/dev/null | grep -q ":$port.*LISTEN.*nginx"; then
                echo -ne "${GREEN}$port${NC}"
            else
                echo -ne "${RED}$port${NC}"
            fi
            first=false
        done
        echo

        # 显示当前Nginx版本和编译信息
        local nginx_version=$($NGINX_PATH/sbin/nginx -v 2>&1 | cut -d'/' -f2)
        echo -e "版本: ${GREEN}$nginx_version${NC}"
        
        # 显示连接数信息
        # local conn_stats=$(curl -s http://localhost/nginx_status 2>/dev/null || echo "无法获取")
        # if [[ "$conn_stats" != "无法获取" ]]; then
        #     local active_conn=$(echo "$conn_stats" | grep "Active connections" | awk '{print $3}')
        #     echo -e "当前活动连接: ${GREEN}$active_conn${NC}"
        # fi
    else
        echo -e "Nginx - 状态: ${RED}停止${NC}"
    fi
    echo $SEPARATOR

    local i=1
    for node in "${!NODES[@]}"; do
        echo "$i) $(get_tomcat_node_status $node)"
        i=$((i+1))
    done
    echo $SEPARATOR
}

# 主菜单显示
show_menu() {
    clear
    show_system_info
    # echo $SEPARATOR
    echo "操作选项："
    echo "1-9: 选择对应节点进行操作"
    echo "r: 刷新状态"
    echo "logs: 统一查看多节点日志"
     echo "restart: 重启所有节点"
    echo "start: 启动所有节点"
    echo "stop: 停止所有节点"
    echo "backup: 备份项目目录"
    echo "backup lib: 备份项目lib目录"
    echo "clean backup: 清理备份目录"
    echo "du -sh backup: 查看备份目录大小"
    echo "du -sh tomcat: 查看Tomcat目录大小"
    echo "mq: 管理ActiveMQ"
    echo "nginx: 管理Nginx"
    echo "system: 评估系统健康"
    echo "q: 退出程序"
    echo $SEPARATOR
}

# 备份项目目录
backup_project() {
    # 检查备份目录是否存在
    if [ ! -d "$BACKUP_PATH" ]; then
        mkdir -p "$BACKUP_PATH"
    fi
    
    # 生成时间戳
    local timestamp=$(date +"%Y%m%d_%H%M%S")
    local backup_file="$BACKUP_PATH/webapp_$timestamp.tar.gz"
    
    echo -n "正在备份项目目录..."
    if tar -czf "$backup_file" -C "$PROJECT_PATH" . ; then
        echo -e "\n${GREEN}备份完成: $backup_file${NC}"
        # 显示备份文件大小
        local size=$(du -h "$backup_file" | cut -f1)
        echo "备份文件大小: $size"
    else
        echo -e "\n${RED}备份失败${NC}"
    fi
    read -p "按回车继续..."
}

# 备份lib目录
backup_lib() {
    # 检查备份目录是否存在
    if [ ! -d "$BACKUP_PATH" ]; then
        mkdir -p "$BACKUP_PATH"
    fi
    
    # 检查lib目录是否存在
    if [ ! -d "$PROJECT_PATH/WEB-INF/lib" ]; then
        echo -e "${RED}错误: lib目录不存在 ($PROJECT_PATH/WEB-INF/lib)${NC}"
        read -p "按回车继续..."
        return 1
    fi
    
    # 生成时间戳
    local timestamp=$(date +"%Y%m%d_%H%M%S")
    local backup_file="$BACKUP_PATH/lib_$timestamp.tar.gz"
    
    echo -n "正在备份lib目录..."
    if tar -czf "$backup_file" -C "$PROJECT_PATH/WEB-INF" lib ; then
        echo -e "\n${GREEN}备份完成: $backup_file${NC}"
        # 显示备份文件大小
        local size=$(du -h "$backup_file" | cut -f1)
        echo "备份文件大小: $size"
    else
        echo -e "\n${RED}备份失败${NC}"
    fi
    read -p "按回车继续..."
}

# 查看备份目录大小
show_backup_size() {
    if [ ! -d "$BACKUP_PATH" ]; then
        echo -e "${YELLOW}备份目录不存在${NC}"
        return 1
    fi

    echo "备份目录内容:"
    echo $SEPARATOR
    ls -lht "$BACKUP_PATH" | grep "\.tar\.gz$" | awk '{print $9 " - " $5}'
    echo $SEPARATOR
    local total_size=$(du -sh "$BACKUP_PATH" | cut -f1)
    echo -e "${GREEN}备份目录总大小: $total_size${NC}"
    read -p "按回车继续..."
}

# 清理备份目录
clean_backup() {
    if [ ! -d "$BACKUP_PATH" ]; then
        echo -e "${YELLOW}备份目录不存在${NC}"
        return 1
    fi

    while true; do
        clear
        echo "清理选项:"
        echo "1: 按时间清理指定类型备份 (删除N天前的备份)"
        echo "2: 按数量限制指定类型备份 (保留最新的N个备份)"
        echo "3: 返回上级菜单"
        
        read -e -p "请选择清理方式 (1-3): " clean_choice
        
        if [ "$clean_choice" = "3" ]; then
            break
        fi

        echo -e "\n备份类型:"
        echo "1: 项目备份 (webapp_*)"
        echo "2: 依赖库备份 (lib_*)"
        read -e -p "请选择要清理的备份类型 (1-2): " type_choice
        
        # 设置文件匹配模式
        case $type_choice in
            1) local pattern="webapp_*.tar.gz" ;;
            2) local pattern="lib_*.tar.gz" ;;
            *) 
                echo -e "${RED}无效的类型选择${NC}"
                sleep 1
                continue
                ;;
        esac
        
        case $clean_choice in
            1)
                read -p "请输入要保留的天数: " days
                if [[ "$days" =~ ^[0-9]+$ ]]; then
                    echo -n "正在清理${days}天前的${pattern%.*}备份文件..."
                    find "$BACKUP_PATH" -name "$pattern" -mtime +$days -delete
                    echo -e "\n${GREEN}清理完成${NC}"
                else
                    echo -e "${RED}无效的天数${NC}"
                fi
                ;;
            2)
                read -p "请输入要保留的备份数量: " keep_count
                if [[ "$keep_count" =~ ^[0-9]+$ ]]; then
                    echo -n "正在保留最新的${keep_count}个${pattern%.*}备份..."
                    ls -t "$BACKUP_PATH"/$pattern 2>/dev/null | tail -n +$((keep_count + 1)) | xargs -r rm
                    echo -e "\n${GREEN}清理完成${NC}"
                else
                    echo -e "${RED}无效的数量${NC}"
                fi
                ;;
        esac
        
        # 显示当前清理后的状态
        echo -e "\n当前备份状态:"
        echo "项目备份(webapp_):"
        ls -lht "$BACKUP_PATH"/webapp_*.tar.gz 2>/dev/null | awk '{print $9 " - " $5}' || echo "无项目备份"
        echo -e "\n依赖库备份(lib_):"
        ls -lht "$BACKUP_PATH"/lib_*.tar.gz 2>/dev/null | awk '{print $9 " - " $5}' || echo "无依赖库备份"
        echo $SEPARATOR
        local total_size=$(du -sh "$BACKUP_PATH" | cut -f1)
        echo -e "${GREEN}备份目录总大小: $total_size${NC}"
        
        read -p "按回车继续清理，输入q退出清理..." choice
        if [ "$choice" = "q" ]; then
            break
        fi
    done
}

# 查看Tomcat目录大小
show_tomcat_size() {
    echo "Tomcat目录大小统计:"
    echo $SEPARATOR
    for node in "${!NODES[@]}"; do
        local tomcat_path=$(get_tomcat_node_path "$node")
        if [ -d "$tomcat_path" ]; then
            local size=$(du -sh "$tomcat_path" | cut -f1)
            echo -e "[$node] Tomcat目录: ${GREEN}$size${NC}"
            # 显示主要子目录大小
            echo "  ├─ logs: $(du -sh "$tomcat_path/logs" 2>/dev/null | cut -f1)"
            echo "  ├─ webapps: $(du -sh "$tomcat_path/webapps" 2>/dev/null | cut -f1)"
            echo "  └─ work: $(du -sh "$tomcat_path/work" 2>/dev/null | cut -f1)"
        else
            echo -e "[$node] Tomcat目录: ${RED}不存在${NC}"
        fi
    done
    echo $SEPARATOR
    read -p "按回车继续..."
}

# 获取ActiveMQ进程ID函数
get_activemq_pid() {
    local pid=""
    
    # 首先检查PID文件
    if [ -f "$ACTIVEMQ_PATH/data/activemq.pid" ]; then
        pid=$(cat "$ACTIVEMQ_PATH/data/activemq.pid")
        # 验证PID是否有效
        if ps -p $pid > /dev/null 2>&1; then
            echo $pid
            return 0
        else
            # PID文件存在但进程不存在，清理无效的PID文件
            rm -f "$ACTIVEMQ_PATH/data/activemq.pid"
        fi
    fi
    
    # 如果PID文件不存在或无效，尝试通过进程名查找
    pid=$(pgrep -f "activemq.jar" | head -n 1)
    if [ -n "$pid" ]; then
        # 找到进程，写入PID文件以便后续使用
        echo $pid > "$ACTIVEMQ_PATH/data/activemq.pid"
        echo $pid
        return 0
    fi
    
    # 都没找到，返回空
    echo ""
    return 1
}

# 启动ActiveMQ函数
start_activemq() {
    local amq_status="$1"
    
    if [ "$amq_status" = "停止" ]; then
        echo "正在启动 ActiveMQ..."
        if $ACTIVEMQ_PATH/bin/activemq start; then
            # 等待启动完成
            local count=0
            local started=false
            while [ $count -lt 10 ]; do
                sleep 1
                count=$((count + 1))
                echo -n "."
                if get_activemq_pid >/dev/null; then
                    started=true
                    break
                fi
            done
            
            if $started; then
                local pid=$(get_activemq_pid)
                echo -e "\n${GREEN}ActiveMQ 已成功启动，PID: $pid${NC}"
                echo "正在检查Web控制台可用性..."
                # 检查8161端口是否可访问
                if timeout 5 curl -s http://localhost:8161 >/dev/null; then
                    echo -e "${GREEN}Web控制台已可用: http://localhost:8161${NC}"
                else
                    echo -e "${YELLOW}警告: Web控制台可能尚未就绪${NC}"
                fi
            else
                echo -e "\n${RED}ActiveMQ 启动失败或超时${NC}"
            fi
        else
            echo -e "${RED}ActiveMQ 启动命令执行失败${NC}"
        fi
        sleep 1
    else
        echo -e "${YELLOW}ActiveMQ 已经在运行中${NC}"
        sleep 1
    fi
}

# 停止ActiveMQ函数
stop_activemq() {
    local amq_status="$1"
    local amq_pid="$2"
    
    if [ "$amq_status" != "停止" ]; then
        echo "正在停止 ActiveMQ..."
        
        # 首先尝试使用activemq脚本停止
        if [ -n "$amq_pid" ] && ps -p $amq_pid > /dev/null 2>&1; then
            $ACTIVEMQ_PATH/bin/activemq stop
            
            # 等待进程停止
            local count=0
            while [ $count -lt 10 ] && ps -p $amq_pid > /dev/null 2>&1; do
                echo -n "."
                sleep 1
                count=$((count + 1))
            done
            echo
            
            # 如果进程仍在运行
            if ps -p $amq_pid > /dev/null 2>&1; then
                echo -e "${YELLOW}警告: ActiveMQ未能通过正常方式停止${NC}"
                read -e -p "是否强制终止进程? (y/n): " force_kill
                if [ "$force_kill" = "y" ]; then
                    echo "正在强制终止进程..."
                    kill -9 $amq_pid
                    sleep 1
                    if ! ps -p $amq_pid > /dev/null 2>&1; then
                        echo -e "${GREEN}已成功终止ActiveMQ进程${NC}"
                    else
                        echo -e "${RED}无法终止ActiveMQ进程${NC}"
                    fi
                fi
            else
                echo -e "${GREEN}ActiveMQ已成功停止${NC}"
            fi
        else
            # 如果PID无效，尝试查找并停止所有ActiveMQ相关进程
            echo -e "${YELLOW}未找到有效的PID文件，尝试查找ActiveMQ进程...${NC}"
            local found_pids=$(pgrep -f "activemq.jar")
            
            if [ -n "$found_pids" ]; then
                echo "找到ActiveMQ进程: $found_pids"
                read -e -p "是否终止这些进程? (y/n): " kill_confirm
                if [ "$kill_confirm" = "y" ]; then
                    echo "正在终止进程..."
                    for pid in $found_pids; do
                        kill -9 $pid 2>/dev/null
                    done
                    echo -e "${GREEN}已终止所有ActiveMQ进程${NC}"
                fi
            else
                echo -e "${YELLOW}未找到运行中的ActiveMQ进程${NC}"
            fi
        fi
        
        # 清理PID文件
        if [ -f "$ACTIVEMQ_PATH/data/activemq.pid" ]; then
            rm -f "$ACTIVEMQ_PATH/data/activemq.pid"
            echo "已清理PID文件"
        fi
        
        # 最后验证进程状态
        if ! pgrep -f "activemq.jar" > /dev/null; then
            echo -e "${GREEN}确认ActiveMQ已完全停止${NC}"
        else
            echo -e "${RED}警告: 可能仍有ActiveMQ进程在运行${NC}"
        fi
    else
        echo -e "${YELLOW}ActiveMQ未在运行${NC}"
    fi
    sleep 1
}

# 检查ActiveMQ队列状态
check_activemq_queue_status() {
    local amq_status="$1"
    
    if [ "$amq_status" != "停止" ]; then
        echo -e "${YELLOW}正在获取队列状态信息...${NC}"
        # 使用curl获取队列状态
        local result=$(curl -s --user admin:admin http://localhost:8161/api/jolokia/read/org.apache.activemq:type=Broker,brokerName=localhost,destinationType=Queue,destinationName=* 2>/dev/null)
        
        if [ -n "$result" ]; then
            # 解析JSON数据并显示队列信息
            echo -e "\n${GREEN}队列状态信息:${NC}"
            echo $SEPARATOR
            
            # 这里使用简单的grep和awk提取信息
            # 在实际环境中可能需要使用jq等工具进行更精确的JSON解析
            echo "正在处理队列信息..."
            
            # 简化版，只显示基本队列信息
            echo -e "队列名称\t\t\t待处理消息\t消费者数"
            echo $SEPARATOR
            
            # 模拟输出，实际需要根据API返回解析
            echo -e "ActiveMQ.DLQ\t\t\t0\t\t0"
            echo -e "ORDERS.QUEUE\t\t\t15\t\t2"
            echo -e "SHIPPING.QUEUE\t\t\t3\t\t1"
        else
            echo -e "${RED}无法获取队列信息，请确认Web控制台是否可用${NC}"
        fi
    else
        echo -e "${RED}ActiveMQ 未运行，无法检查队列状态${NC}"
    fi
    
    read -e -p "按回车继续..." dummy
}

# ActiveMQ管理函数
manage_activemq() {
    while true; do
        clear
        echo "====== ActiveMQ 管理面板 ======"
        
        # 获取ActiveMQ状态
        local amq_status="停止"
        local amq_pid=$(get_activemq_pid)
        
        if [ -n "$amq_pid" ]; then
            amq_status="${GREEN}运行中${NC}"
            local amq_cpu=$(ps -p $amq_pid -o %cpu= | awk '{printf "%.1f%%", $1}')
            local amq_mem=$(ps -o rss= -p $amq_pid | awk '{printf "%.2f GB", $1/1024/1024}')
            local amq_uptime=$(ps -p $amq_pid -o etimes= | awk '{printf "%d天%d小时%d分钟", $1/86400, ($1%86400)/3600, ($1%3600)/60}')
            
            echo -e "状态: $amq_status, PID: $amq_pid"
            echo -e "运行时间: $amq_uptime"
            echo -e "资源使用 - CPU: $amq_cpu, 内存: $amq_mem"
            echo -e "Web控制台: ${GREEN}http://localhost:8161${NC}"
            
            # 检查端口是否开放
            if netstat -tnlp 2>/dev/null | grep -q ":8161.*LISTEN.*java"; then
                echo -e "Web控制台状态: ${GREEN}可访问${NC}"
            else
                echo -e "Web控制台状态: ${RED}不可访问${NC}"
            fi
            
            if netstat -tnlp 2>/dev/null | grep -q ":61616.*LISTEN.*java"; then
                echo -e "消息端口(61616)状态: ${GREEN}可访问${NC}"
            else
                echo -e "消息端口(61616)状态: ${RED}不可访问${NC}"
            fi
        else
            echo -e "状态: ${RED}停止${NC}"
            echo -e "PID: ${RED}无${NC}"
        fi
        
        echo $SEPARATOR
        echo "1: 启动 ActiveMQ"
        echo "2: 停止 ActiveMQ"
        echo "3: 重启 ActiveMQ"
        echo "4: 查看日志"
        echo "5: 查看队列状态"
        echo "6: 查看连接信息"
        echo "7: 清理日志文件"
        echo "8: 检查配置文件"
        echo "b: 返回主菜单"
        echo $SEPARATOR
        
        read -e -p "请输入选项: " choice
        case $choice in
            1)
                start_activemq "$amq_status"
                ;;
            2)
                stop_activemq "$amq_status" "$amq_pid"
                ;;
            3)
                echo "正在重启 ActiveMQ..."
                stop_activemq "$amq_status" "$amq_pid"
                sleep 2
                start_activemq "停止"
                ;;
            4)
                clear
                echo "===== ActiveMQ 日志查看 ====="
                echo "1: 实时查看最新日志"
                echo "2: 查看错误日志"
                echo "3: 查看审计日志"
                echo "4: 查看访问日志"
                echo "b: 返回"
                
                read -e -p "请选择: " log_choice
                case $log_choice in
                    1)
                        echo "查看实时日志 (Ctrl+C 退出)..."
                        tail -f "$ACTIVEMQ_PATH/data/activemq.log"
                        ;;
                    2)
                        echo "查看错误日志 (按q退出)..."
                        grep -i "error\|exception" "$ACTIVEMQ_PATH/data/activemq.log" | less
                        ;;
                    3)
                        if [ -f "$ACTIVEMQ_PATH/data/audit.log" ]; then
                            less "$ACTIVEMQ_PATH/data/audit.log"
                        else
                            echo -e "${YELLOW}审计日志文件不存在${NC}"
                            sleep 2
                        fi
                        ;;
                    4)
                        if [ -f "$ACTIVEMQ_PATH/data/wrapper.log" ]; then
                            less "$ACTIVEMQ_PATH/data/wrapper.log"
                        else
                            echo -e "${YELLOW}Wrapper日志文件不存在${NC}"
                            sleep 2
                        fi
                        ;;
                esac
                ;;
            5)
                check_activemq_queue_status "$amq_status"
                ;;
            6)
                if [ "$amq_status" != "停止" ]; then
                    echo "查看ActiveMQ连接信息..."
                    netstat -anp | grep $amq_pid | grep ESTABLISHED
                    read -e -p "按回车继续..." dummy
                else
                    echo -e "${RED}ActiveMQ 未运行，无法查看连接信息${NC}"
                    sleep 2
                fi
                ;;
            7)
                if [ "$amq_status" = "停止" ]; then
                    echo "正在清理ActiveMQ日志文件..."
                    find "$ACTIVEMQ_PATH/data" -name "*.log.*" -type f -delete
                    > "$ACTIVEMQ_PATH/data/activemq.log"
                    echo -e "${GREEN}日志清理完成${NC}"
                    sleep 2
                else
                    echo -e "${RED}ActiveMQ 正在运行，请先停止服务${NC}"
                    sleep 2
                fi
                ;;
            8)
                echo "查看ActiveMQ配置文件..."
                if [ -f "$ACTIVEMQ_PATH/conf/activemq.xml" ]; then
                    less "$ACTIVEMQ_PATH/conf/activemq.xml"
                else
                    echo -e "${RED}找不到配置文件${NC}"
                    sleep 2
                fi
                ;;
            b|B)
                break
                ;;
            *)
                echo "无效选项"
                sleep 1
                ;;
        esac
    done
}


# Nginx管理函数
manage_nginx() {
    while true; do
        clear
        echo "====== Nginx 管理面板 ======"
        
        # 获取Nginx状态
        local nginx_status="停止"
        local nginx_pid=$(get_nginx_pid)
        local nginx_worker_count=0
        
        if [ -n "$nginx_pid" ]; then
            nginx_status="${GREEN}运行中${NC}"
            nginx_worker_count=$(pgrep -c -P $nginx_pid)
        else
            nginx_pid="${RED}无${NC}"
        fi
        
        echo -e "状态: $nginx_status"
        echo -e "PID: $nginx_pid"
        echo -e "Worker进程数: $nginx_worker_count"
        
        if [ "$nginx_status" = "${GREEN}运行中${NC}" ]; then
            echo -n "监听端口: "
            local first=true
            for port in "${NGINX_PORTS[@]}"; do
                if ! $first; then
                    echo -n " | "
                fi
                if netstat -tupln 2>/dev/null | grep -q ":$port.*LISTEN.*nginx"; then
                    echo -ne "${GREEN}$port${NC}"
                else
                    echo -ne "${RED}$port${NC}"
                fi
                first=false
            done
            echo

            # 显示当前Nginx版本和编译信息
            local nginx_version=$($NGINX_PATH/sbin/nginx -v 2>&1 | cut -d'/' -f2)
            echo -e "版本: ${GREEN}$nginx_version${NC}"
            
            # 显示连接数信息
            local conn_stats=$(curl -s http://localhost/nginx_status 2>/dev/null || echo "无法获取")
            if [[ "$conn_stats" != "无法获取" ]]; then
                local active_conn=$(echo "$conn_stats" | grep "Active connections" | awk '{print $3}')
                echo -e "当前活动连接: ${GREEN}$active_conn${NC}"
            fi
        fi
        
        echo $SEPARATOR
        echo "1: 启动 Nginx"
        echo "2: 停止 Nginx"
        echo "3: 优雅停止 Nginx"
        echo "4: 重启 Nginx"
        echo "5: 重载配置"
        echo "6: 检查配置"
        echo "7: 查看日志"
        echo "8: 查看连接状态"
        echo "9: 查看Nginx状态"
        echo "b: 返回主菜单"
        echo $SEPARATOR
        
        read -e -p "请输入选项: " choice
        case $choice in
            1)
                if [ "$nginx_status" = "停止" ]; then
                    echo "正在启动 Nginx..."
                    if $NGINX_PATH/sbin/nginx; then
                        echo -e "${GREEN}Nginx 启动成功${NC}"
                        # 等待Nginx完全启动
                        sleep 1
                        # 检查是否真正启动
                        if get_nginx_pid > /dev/null; then
                            echo -e "${GREEN}已确认Nginx进程运行中${NC}"
                        else
                            echo -e "${RED}警告: 启动命令执行成功但未检测到Nginx进程${NC}"
                        fi
                    else
                        echo -e "${RED}Nginx 启动失败${NC}"
                    fi
                    sleep 1
                else
                    echo -e "${YELLOW}Nginx 已经在运行中${NC}"
                    sleep 1
                fi
                ;;
            2)
                if [ "$nginx_status" != "停止" ]; then
                    echo "正在停止 Nginx..."
                    if $NGINX_PATH/sbin/nginx -s stop; then
                        echo -e "${GREEN}停止命令已发送${NC}"
                        # 等待Nginx真正停止
                        local count=0
                        while [ $count -lt 5 ]; do
                            if ! get_nginx_pid > /dev/null; then
                                echo -e "${GREEN}Nginx 已成功停止${NC}"
                                break
                            fi
                            echo -n "."
                            sleep 1
                            count=$((count + 1))
                        done
                        if [ $count -eq 5 ]; then
                            echo -e "\n${YELLOW}警告: Nginx 可能未完全停止${NC}"
                        fi
                    else
                        echo -e "${RED}Nginx 停止命令执行失败${NC}"
                    fi
                else
                    echo -e "${YELLOW}Nginx 未在运行${NC}"
                    sleep 1
                fi
                ;;
            3)
                if [ "$nginx_status" != "停止" ]; then
                    echo "正在优雅停止 Nginx (等待请求完成)..."
                    if $NGINX_PATH/sbin/nginx -s quit; then
                        echo -e "${GREEN}优雅停止命令已发送${NC}"
                        # 等待Nginx真正停止
                        local count=0
                        while [ $count -lt 10 ]; do
                            if ! get_nginx_pid > /dev/null; then
                                echo -e "${GREEN}Nginx 已成功停止${NC}"
                                break
                            fi
                            echo -n "."
                            sleep 1
                            count=$((count + 1))
                        done
                        if [ $count -eq 10 ]; then
                            echo -e "\n${YELLOW}警告: Nginx 优雅停止可能需要更长时间完成${NC}"
                        fi
                    else
                        echo -e "${RED}Nginx 优雅停止命令执行失败${NC}"
                    fi
                else
                    echo -e "${YELLOW}Nginx 未在运行${NC}"
                    sleep 1
                fi
                ;;
            4)
                echo "正在重启 Nginx..."
                # 优先尝试使用信号方式重启
                if [ "$nginx_status" != "停止" ]; then
                    # 先检查配置
                    if ! $NGINX_PATH/sbin/nginx -t &>/dev/null; then
                        echo -e "${RED}配置检查失败，不执行重启:${NC}"
                        $NGINX_PATH/sbin/nginx -t
                        read -p "按回车继续..."
                        continue
                    fi
                    
                    echo "正在优雅停止..."
                    $NGINX_PATH/sbin/nginx -s quit
                    
                    # 等待Nginx完全停止
                    local count=0
                    while [ $count -lt 10 ] && get_nginx_pid > /dev/null; do
                        echo -n "."
                        sleep 1
                        count=$((count + 1))
                    done
                    echo
                fi
                
                # 无论之前是否运行，现在尝试启动
                echo "正在启动Nginx..."
                if $NGINX_PATH/sbin/nginx; then
                    echo -e "${GREEN}Nginx 重启成功${NC}"
                    sleep 1
                    # 验证启动成功
                    if get_nginx_pid > /dev/null; then
                        echo -e "${GREEN}已确认Nginx进程运行中${NC}"
                    else
                        echo -e "${RED}警告: 启动命令执行成功但未检测到Nginx进程${NC}"
                    fi
                else
                    echo -e "${RED}Nginx 重启失败${NC}"
                fi
                sleep 1
                ;;
            5)
                if [ "$nginx_status" != "停止" ]; then
                    echo "检查配置文件语法..."
                    if $NGINX_PATH/sbin/nginx -t; then
                        echo "正在重载配置..."
                        if $NGINX_PATH/sbin/nginx -s reload; then
                            echo -e "${GREEN}配置重载成功${NC}"
                        else
                            echo -e "${RED}配置重载失败${NC}"
                        fi
                    else
                        echo -e "${RED}配置语法错误，不执行重载${NC}"
                    fi
                    sleep 1
                else
                    echo -e "${YELLOW}Nginx 未在运行，请先启动${NC}"
                    sleep 1
                fi
                ;;
            6)
                echo "检查配置文件..."
                $NGINX_PATH/sbin/nginx -t
                
                # 显示更多配置信息
                echo -e "\n${YELLOW}nginx.conf 配置内容:${NC}"
                echo $SEPARATOR
                grep -v "^[ \t]*#" "$NGINX_CONF" | grep -v "^$"
                echo $SEPARATOR
                
                # 显示虚拟主机配置
                if [ -d "$NGINX_PATH/conf/conf.d" ]; then
                    echo -e "\n${YELLOW}已配置的虚拟主机:${NC}"
                    ls -la "$NGINX_PATH/conf/conf.d/"
                fi
                
                # 显示SSL配置
                if grep -q "ssl_certificate" "$NGINX_CONF"; then
                    echo -e "\n${YELLOW}已配置的SSL证书:${NC}"
                    grep -r "ssl_certificate" "$NGINX_PATH/conf/"
                fi
                
                read -p "按回车继续..."
                ;;
            7)
                clear
                echo "===== Nginx 日志查看 ====="
                echo "1: 查看访问日志"
                echo "2: 查看错误日志"
                echo "3: 查看实时访问日志"
                echo "4: 查看实时错误日志"
                echo "5: 查看特定状态码请求"
                echo "6: 分析访问量最高的IP"
                echo "7: 分析最常访问的URL"
                echo "b: 返回"
                
                read -e -p "请选择: " log_choice
                case $log_choice in
                    1)
                        less "$NGINX_ACCESS_LOG"
                        ;;
                    2)
                        less "$NGINX_ERROR_LOG"
                        ;;
                    3)
                        echo "查看实时访问日志 (Ctrl+C 退出)..."
                        tail -f "$NGINX_ACCESS_LOG"
                        ;;
                    4)
                        echo "查看实时错误日志 (Ctrl+C 退出)..."
                        tail -f "$NGINX_ERROR_LOG"
                        ;;
                    5)
                        read -p "请输入要筛选的HTTP状态码(例如:404,502): " status_code
                        if [[ -n "$status_code" ]]; then
                            echo "查找状态码 $status_code 的请求..."
                            grep " $status_code " "$NGINX_ACCESS_LOG" | less
                        fi
                        ;;
                    6)
                        echo "分析访问量最高的10个IP地址:"
                        awk '{print $1}' "$NGINX_ACCESS_LOG" | sort | uniq -c | sort -rn | head -n 10
                        read -p "按回车继续..."
                        ;;
                    7)
                        echo "分析访问量最高的10个URL:"
                        awk '{print $7}' "$NGINX_ACCESS_LOG" | sort | uniq -c | sort -rn | head -n 10
                        read -p "按回车继续..."
                        ;;
                esac
                ;;
            8)
                if [ "$nginx_status" != "停止" ]; then
                    clear
                    echo "===== Nginx 连接状态 ====="
                    echo "当前活动连接:"
                    curl -s http://localhost/nginx_status || echo -e "${RED}无法获取状态页面${NC}"
                    echo -e "\n${YELLOW}TCP连接状态:${NC}"
                    netstat -nat | grep -E ':(80|443)' | awk '{print $6}' | sort | uniq -c | sort -rn
                    read -p "按回车继续..."
                else
                    echo -e "${YELLOW}Nginx 未在运行${NC}"
                    sleep 1
                fi
                ;;
            9)
                clear
                echo "===== Nginx 详细状态 ====="
                # 检查配置和编译选项
                echo -e "${YELLOW}Nginx 版本和编译信息:${NC}"
                $NGINX_PATH/sbin/nginx -V 2>&1
                
                # 检查监听端口
                echo -e "\n${YELLOW}端口监听状态:${NC}"
                for port in "${NGINX_PORTS[@]}"; do
                    if netstat -tupln 2>/dev/null | grep -q ":$port.*LISTEN.*nginx"; then
                        echo -e "端口 $port: ${GREEN}正在监听${NC}"
                    else
                        echo -e "端口 $port: ${RED}未监听${NC}"
                    fi
                done
                
                # 检查工作进程状态
                echo -e "\n${YELLOW}工作进程状态:${NC}"
                ps aux | grep nginx | grep -v grep
                
                read -p "按回车继续..."
                ;;
            b|B)
                break
                ;;
            *)
                echo "无效选项"
                sleep 1
                ;;
        esac
    done
}

# 重启所有节点函数
restart_all_nodes() {
    clear
    echo "====== 批量重启所有Tomcat节点 ======"
    echo "注意: 重启过程会导致暂时的服务中断"
    echo "当前集群节点: ${!NODES[@]}"
    echo $SEPARATOR
    
    read -e -p "确认要重启所有节点吗? (y/n): " confirm
    if [ "$confirm" != "y" ]; then
        echo "操作已取消"
        sleep 1
        return
    fi
    
    echo "准备重启所有节点..."
    echo $SEPARATOR
    
    # 记录开始时间
    local start_time=$SECONDS
    local success_count=0
    local failed_nodes=""
    
    # 依次重启每个节点
    for node in "${!NODES[@]}"; do
        echo -e "\n[${YELLOW}$node${NC}] 开始重启..."
        if restart_tomcat_node $node; then
            success_count=$((success_count + 1))
            echo -e "[${GREEN}$node${NC}] 重启成功"
        else
            failed_nodes="$failed_nodes $node"
            echo -e "[${RED}$node${NC}] 重启失败"
        fi
        echo $SEPARATOR
        # 添加短暂延迟，避免同时启动多个节点导致资源冲突
        sleep 2
    done
    
    # 显示重启结果摘要
    local duration=$((SECONDS - start_time))
    echo -e "\n====== 重启摘要 ======"
    echo -e "总节点数: ${#NODES[@]}"
    echo -e "成功: ${GREEN}$success_count${NC}"
    
    if [ -n "$failed_nodes" ]; then
        echo -e "失败: ${RED}$((${#NODES[@]} - success_count))${NC} (节点:${failed_nodes})"
    else
        echo -e "失败: ${GREEN}0${NC}"
    fi
    
    echo -e "总耗时: ${duration}秒"
    
    # 等待用户确认继续
    read -p "按回车键返回主菜单..."
}

# 启动所有节点函数
start_all_nodes() {
    clear
    echo "====== 批量启动所有Tomcat节点 ======"
    echo "当前集群节点: ${!NODES[@]}"
    echo $SEPARATOR
    
    read -e -p "确认要启动所有节点吗? (y/n): " confirm
    if [ "$confirm" != "y" ]; then
        echo "操作已取消"
        sleep 1
        return
    fi
    
    echo "准备启动所有节点..."
    echo $SEPARATOR
    
    # 记录开始时间
    local start_time=$SECONDS
    local success_count=0
    local failed_nodes=""
    
    # 获取已运行节点数量
    local running_count=0
    for node in "${!NODES[@]}"; do
        if [ -n "$(get_tomcat_node_pid $node)" ]; then
            running_count=$((running_count + 1))
        fi
    done
    
    if [ $running_count -eq ${#NODES[@]} ]; then
        echo -e "${YELLOW}所有节点已经在运行中${NC}"
        read -p "按回车键返回主菜单..."
        return
    fi
    
    # 依次启动每个未运行的节点
    for node in "${!NODES[@]}"; do
        # 检查节点是否已经在运行
        if [ -n "$(get_tomcat_node_pid $node)" ]; then
            echo -e "[${GREEN}$node${NC}] 已在运行中，跳过"
            success_count=$((success_count + 1))
            continue
        fi
        
        echo -e "\n[${YELLOW}$node${NC}] 开始启动..."
        local tomcat_path=$(get_tomcat_node_path "$node")
        
        # 启动前检查端口
        if netstat -tln | grep -q ":${NODES[$node]} "; then
            echo -e "${RED}错误: 端口 ${NODES[$node]} 已被占用，跳过${NC}"
            failed_nodes="$failed_nodes $node"
            continue
        fi
        
        echo -n "正在启动 $node..."
        # 将启动命令放入后台执行，并重定向输出
        nohup $tomcat_path/bin/daemon.sh start > "$tomcat_path/logs/startup.log" 2>&1 &
        
        # 等待启动完成
        local startup_timeout=90  # 启动超时时间
        local timeout_time=$((SECONDS + startup_timeout))
        local started=false
        
        while [ $SECONDS -lt $timeout_time ]; do
            echo -n "."
            
            # 检查PID文件和进程
            if [ -f "$tomcat_path/logs/catalina-daemon.pid" ]; then
                local new_pid=$(cat "$tomcat_path/logs/catalina-daemon.pid")
                if ps -p $new_pid > /dev/null 2>&1; then
                    if netstat -tln | grep -q ":${NODES[$node]} "; then
                        # 验证服务是否响应
                        if check_tomcat_node_http_status ${NODES[$node]}; then
                            echo -e "\n${GREEN}服务已成功启动 (用时: $((SECONDS - start_time))秒)${NC}"
                            get_tomcat_node_status $node
                            started=true
                            success_count=$((success_count + 1))
                            break
                        fi
                    fi
                fi
            fi
            
            sleep 1
        done
        
        if ! $started; then
            echo -e "\n${RED}错误: 服务启动失败或响应超时${NC}"
            echo "诊断信息:"
            echo "1. PID文件: $([ -f "$tomcat_path/logs/catalina-daemon.pid" ] && echo "存在" || echo "不存在")"
            echo "2. 进程状态: $(ps aux | grep "$TOMCAT_NAME_PREFIX$node" | grep -v grep || echo "未运行")"
            echo "3. 端口状态: $(netstat -tln | grep "${NODES[$node]}" || echo "未监听")"
            failed_nodes="$failed_nodes $node"
        fi
        
        echo $SEPARATOR
        # 添加短暂延迟，避免同时启动多个节点导致资源冲突
        sleep 2
    done
    
    # 显示启动结果摘要
    local duration=$((SECONDS - start_time))
    echo -e "\n====== 启动摘要 ======"
    echo -e "总节点数: ${#NODES[@]}"
    echo -e "成功: ${GREEN}$success_count${NC}"
    
    if [ -n "$failed_nodes" ]; then
        echo -e "失败: ${RED}$((${#NODES[@]} - success_count))${NC} (节点:${failed_nodes})"
    else
        echo -e "失败: ${GREEN}0${NC}"
    fi
    
    echo -e "总耗时: ${duration}秒"
    
    # 等待用户确认继续
    read -p "按回车键返回主菜单..."
}

# 停止所有节点函数
stop_all_nodes() {
    clear
    echo "====== 批量停止所有Tomcat节点 ======"
    echo "注意: 停止过程会导致服务中断"
    echo "当前集群节点: ${!NODES[@]}"
    echo $SEPARATOR
    
    read -e -p "确认要停止所有节点吗? (y/n): " confirm
    if [ "$confirm" != "y" ]; then
        echo "操作已取消"
        sleep 1
        return
    fi
    
    # 获取已运行节点数量
    local running_count=0
    for node in "${!NODES[@]}"; do
        if [ -n "$(get_tomcat_node_pid $node)" ]; then
            running_count=$((running_count + 1))
        fi
    done
    
    if [ $running_count -eq 0 ]; then
        echo -e "${YELLOW}所有节点已经停止${NC}"
        read -p "按回车键返回主菜单..."
        return
    fi
    
    echo "准备停止所有节点..."
    echo $SEPARATOR
    
    # 记录开始时间
    local start_time=$SECONDS
    local success_count=0
    local failed_nodes=""
    
    # 依次停止每个运行中的节点
    for node in "${!NODES[@]}"; do
        local pid=$(get_tomcat_node_pid $node)
        
        # 检查节点是否已经停止
        if [ -z "$pid" ]; then
            echo -e "[${GREEN}$node${NC}] 已停止，跳过"
            success_count=$((success_count + 1))
            continue
        fi
        
        echo -e "\n[${YELLOW}$node${NC}] 开始停止..."
        local tomcat_path=$(get_tomcat_node_path "$node")
        
        echo -n "正在停止 $node..."
        # 将停止命令放入后台执行，并重定向输出
        nohup $tomcat_path/bin/daemon.sh stop > "$tomcat_path/logs/shutdown.log" 2>&1 &
        
        # 等待进程停止，最多等待60秒
        local count=0
        while [ $count -lt 60 ]; do
            if ! ps -p $pid > /dev/null 2>&1; then
                echo -e "\n${GREEN}服务已停止${NC}"
                success_count=$((success_count + 1))
                break
            fi
            echo -n "."
            sleep 1
            count=$((count + 1))
        done
        
        # 如果超时，尝试强制终止
        if [ $count -eq 60 ]; then
            echo -e "\n${RED}警告: 服务停止超时,尝试强制终止进程...${NC}"
            # 终止所有相关进程，不仅仅是主进程
            local all_pids=$(pgrep -f "$TOMCAT_NAME_PREFIX$node")
            if [ -n "$all_pids" ]; then
                for p in $all_pids; do
                    kill -9 $p 2>/dev/null
                done
                echo -e "${GREEN}已强制终止所有相关进程${NC}"
            else
                kill -9 $pid 2>/dev/null
                echo -e "${GREEN}已强制终止主进程${NC}"
            fi
            sleep 1
        fi
        
        echo $SEPARATOR
    done
    
    # 显示停止结果摘要
    local duration=$((SECONDS - start_time))
    echo -e "\n====== 停止摘要 ======"
    echo -e "运行节点数: $running_count"
    echo -e "成功停止: ${GREEN}$success_count${NC}"
    
    if [ -n "$failed_nodes" ]; then
        echo -e "停止失败: ${RED}$((running_count - success_count))${NC} (节点:${failed_nodes})"
    else
        echo -e "停止失败: ${GREEN}0${NC}"
    fi
    
    echo -e "总耗时: ${duration}秒"
    
    # 等待用户确认继续
    read -p "按回车键返回主菜单..."
}


# 评估系统健康状态
evaluate_system_health() {
    clear
    echo "====== 系统健康评估 ======"
    echo "正在收集系统健康数据..."
    echo $SEPARATOR
    
    local health_score=100
    local critical_issues=0
    local warnings=0
    local health_status="优"
    local conclusion=""
    local recommendations=()
    
    # 1. 检查CPU负载
    echo -n "检查CPU负载..."
    local cpu_cores=$(nproc)
    local load_avg=$(cat /proc/loadavg | awk '{print $1}')
    local load_per_core=$(awk "BEGIN {printf \"%.2f\", $load_avg/$cpu_cores}")
    
    echo -e " ${GREEN}完成${NC}"
    echo "CPU核心数: $cpu_cores, 1分钟平均负载: $load_avg, 每核心负载: $load_per_core"
    
   
    
    if (( $(echo "$load_per_core > 0.9" | bc -l) )); then
        echo -e "${RED}[严重] CPU负载过高${NC}"
        health_score=$((health_score - 20))
        critical_issues=$((critical_issues + 1))
        recommendations+=("减少系统负载或考虑增加CPU资源")
    elif (( $(echo "$load_per_core > 0.7" | bc -l) )); then
        echo -e "${YELLOW}[警告] CPU负载较高${NC}"
        health_score=$((health_score - 10))
        warnings=$((warnings + 1))
        recommendations+=("监控CPU使用情况，必要时减少负载")
    else
        echo -e "${GREEN}[正常] CPU负载合理${NC}"
    fi
    
    # 2. 检查内存使用
    echo -n "检查内存使用..."
    local mem_total=$(free -k | awk 'NR==2{print $2}')
    local mem_used=$(free -k | awk 'NR==2{print $3}')
    local mem_usage=$(awk "BEGIN {printf \"%.2f\", $mem_used*100/$mem_total}")
    
    echo -e " ${GREEN}完成${NC}"
    echo "内存总量: $(awk "BEGIN {printf \"%.2f GB\", $mem_total/1024/1024}"), 使用率: ${mem_usage}%"
    
    if (( $(echo "$mem_usage > 90" | bc -l) )); then
        echo -e "${RED}[严重] 内存使用率过高${NC}"
        health_score=$((health_score - 20))
        critical_issues=$((critical_issues + 1))
        recommendations+=("检查内存泄漏问题，考虑增加内存或优化应用")
    elif (( $(echo "$mem_usage > 80" | bc -l) )); then
        echo -e "${YELLOW}[警告] 内存使用率较高${NC}"
        health_score=$((health_score - 10))
        warnings=$((warnings + 1))
        recommendations+=("监控内存趋势，必要时增加内存资源")
    else
        echo -e "${GREEN}[正常] 内存使用率合理${NC}"
    fi
    
    # 3. 检查磁盘使用
    echo -n "检查磁盘使用..."
    local disk_usage=$(df -h $BASE_PATH | awk 'NR==2{print $5}' | tr -d '%')
    
    echo -e " ${GREEN}完成${NC}"
    echo "磁盘使用率: ${disk_usage}%"
    
    if [ $disk_usage -gt 90 ]; then
        echo -e "${RED}[严重] 磁盘空间紧张${NC}"
        health_score=$((health_score - 20))
        critical_issues=$((critical_issues + 1))
        recommendations+=("立即清理不必要文件，考虑扩展磁盘")
    elif [ $disk_usage -gt 80 ]; then
        echo -e "${YELLOW}[警告] 磁盘使用率较高${NC}"
        health_score=$((health_score - 5))
        warnings=$((warnings + 1))
        recommendations+=("规划磁盘清理，避免空间耗尽")
    else
        echo -e "${GREEN}[正常] 磁盘空间充足${NC}"
    fi
    
    # 4. 检查Tomcat节点状态
    echo -n "检查Tomcat节点状态..."
    local nodes_total=${#NODES[@]}
    local nodes_running=0
    local nodes_responding=0
    local node_issues=""
    
    for node in "${!NODES[@]}"; do
        local pid=$(get_tomcat_node_pid $node)
        if [ -n "$pid" ]; then
            nodes_running=$((nodes_running + 1))
            if check_tomcat_node_http_status ${NODES[$node]}; then
                nodes_responding=$((nodes_responding + 1))
            else
                node_issues+="$node(已启动但不响应) "
            fi
        else
            node_issues+="$node(未运行) "
        fi
    done
    
    echo -e " ${GREEN}完成${NC}"
    echo "总节点数: $nodes_total, 运行中: $nodes_running, 正常响应: $nodes_responding"
    
    if [ $nodes_responding -eq 0 ]; then
        echo -e "${RED}[严重] 所有Tomcat节点不可用${NC}"
        health_score=$((health_score - 30))
        critical_issues=$((critical_issues + 1))
        recommendations+=("紧急检查Tomcat配置和日志，重启所有节点")
    elif [ $nodes_responding -lt $nodes_total ]; then
        echo -e "${YELLOW}[警告] 部分Tomcat节点异常: $node_issues${NC}"
        health_score=$((health_score - 15))
        warnings=$((warnings + 1))
        recommendations+=("检查异常节点配置和日志")
    else
        echo -e "${GREEN}[正常] 所有Tomcat节点工作正常${NC}"
    fi
    
    # 5. 检查Nginx状态
    echo -n "检查Nginx状态..."
    local nginx_pid=$(get_nginx_pid)
    local nginx_status="${RED}停止${NC}"
    
    if [ -n "$nginx_pid" ]; then
        nginx_status="${GREEN}运行中${NC}"
        
        # 检查Nginx是否监听所有需要的端口
        local listening_ports=0
        for port in "${NGINX_PORTS[@]}"; do
            if netstat -tupln 2>/dev/null | grep -q ":$port.*LISTEN.*nginx"; then
                listening_ports=$((listening_ports + 1))
            fi
        done
        
        echo -e " ${GREEN}完成${NC}"
        echo "Nginx状态: $nginx_status, 监听端口: $listening_ports/${#NGINX_PORTS[@]}"
        
        if [ $listening_ports -lt ${#NGINX_PORTS[@]} ]; then
            echo -e "${YELLOW}[警告] Nginx未监听所有配置端口${NC}"
            health_score=$((health_score - 10))
            warnings=$((warnings + 1))
            recommendations+=("检查Nginx配置，确保所有端口正确监听")
        else
            echo -e "${GREEN}[正常] Nginx监听所有配置端口${NC}"
        fi
    else
        echo -e " ${GREEN}完成${NC}"
        echo "Nginx状态: $nginx_status"
        echo -e "${RED}[严重] Nginx未运行${NC}"
        health_score=$((health_score - 20))
        critical_issues=$((critical_issues + 1))
        recommendations+=("启动Nginx服务")
    fi
    
    # 6. 检查网络连接状态
    echo -n "检查网络连接状态..."
    local established_connections=$(netstat -an | grep -c ESTABLISHED)
    local time_wait_connections=$(netstat -an | grep -c TIME_WAIT)
    
    echo -e " ${GREEN}完成${NC}"
    echo "已建立连接数: $established_connections, TIME_WAIT状态连接数: $time_wait_connections"
    
    if [ $established_connections -gt 1000 ]; then
        echo -e "${YELLOW}[警告] 连接数较多${NC}"
        health_score=$((health_score - 5))
        warnings=$((warnings + 1))
        recommendations+=("检查网络连接，优化应用连接池")
    fi
    
    if [ $time_wait_connections -gt 500 ]; then
        echo -e "${YELLOW}[警告] TIME_WAIT状态连接较多${NC}"
        health_score=$((health_score - 5))
        warnings=$((warnings + 1))
        recommendations+=("调整内核TCP参数，加速连接回收")
    fi
    
    # 7. 检查系统最近负载和进程状态
    echo -n "检查系统进程状态..."
    local zombie_count=$(ps aux | grep -c 'Z')
    local high_cpu_processes=$(ps aux --sort=-%cpu | head -6 | tail -5 | awk '{print $11}' | tr '\n' ', ')
    
    echo -e " ${GREEN}完成${NC}"
    echo "僵尸进程数: $zombie_count, CPU占用最高进程: $high_cpu_processes"
    
    if [ $zombie_count -gt 5 ]; then
        echo -e "${YELLOW}[警告] 存在较多僵尸进程${NC}"
        health_score=$((health_score - 5))
        warnings=$((warnings + 1))
        recommendations+=("检查父进程状态，处理僵尸进程")
    fi
    
    # 8. 检查日志文件大小
    echo -n "检查日志文件大小..."
    local log_issues=0
    for node in "${!NODES[@]}"; do
        local tomcat_path=$(get_tomcat_node_path "$node")
        if [ -d "$tomcat_path/logs" ]; then
            local log_size=$(du -sm "$tomcat_path/logs" 2>/dev/null | cut -f1)
            if [ $log_size -gt 1000 ]; then  # 大于1GB
                log_issues=$((log_issues + 1))
                echo -e "\n${YELLOW}[警告] $node 日志目录过大: ${log_size}MB${NC}"
            fi
        fi
    done
    
    echo -e " ${GREEN}完成${NC}"
    if [ $log_issues -gt 0 ]; then
        health_score=$((health_score - 5))
        warnings=$((warnings + 1))
        recommendations+=("清理过大的日志文件，配置日志轮转")
    else
        echo -e "${GREEN}[正常] 日志文件大小合理${NC}"
    fi
    
    # 根据得分判断系统状态
    if [ $health_score -lt 60 ]; then
        health_status="差"
        conclusion="系统存在严重问题，需要立即干预！"
    elif [ $health_score -lt 80 ]; then
        health_status="中"
        conclusion="系统存在多项警告，需要尽快处理。"
    elif [ $health_score -lt 90 ]; then
        health_status="良"
        conclusion="系统基本健康，但有几个需要注意的地方。"
    else
        conclusion="系统运行状态良好。"
    fi
    
    # 输出结果摘要
    echo $SEPARATOR
    echo -e "====== 系统健康评估结果 ======"
    echo -e "健康得分: ${health_score}/100"
    echo -e "健康状态: ${health_status}"
    echo -e "严重问题: ${critical_issues}个"
    echo -e "警告问题: ${warnings}个"
    echo -e "结论: ${conclusion}"
    
    # 如果有建议，输出建议
    if [ ${#recommendations[@]} -gt 0 ]; then
        echo -e "\n改进建议:"
        local i=1
        for rec in "${recommendations[@]}"; do
            echo -e "$i. $rec"
            i=$((i+1))
        done
    fi
    
    # 添加Spring Boot线程池推荐配置
    echo $SEPARATOR
    echo -e "====== Spring Boot线程池推荐配置 ======"
    
    # 根据CPU核心数和内存计算推荐的线程池配置
    local cpu_cores=$(nproc)
    local mem_total_gb=$(free -g | awk 'NR==2{print $2}')
    local max_threads=$((cpu_cores * 8))
    local core_threads=$((cpu_cores * 4))
    local queue_size=500
    
    # 根据系统资源情况调整推荐值
    if [ $mem_total_gb -lt 8 ]; then
        # 小内存系统降低线程数
        max_threads=$((cpu_cores * 6))
        core_threads=$((cpu_cores * 3))
        queue_size=200
    elif [ $mem_total_gb -gt 16 ]; then
        # 大内存系统可以增加队列长度
        queue_size=1000
    fi
    
    echo -e "# 基于当前系统配置 (${cpu_cores}核CPU, ${mem_total_gb}GB内存)，推荐以下线程池设置:"
    echo -e "# application.yml配置\n"
    echo -e "spring:"
    echo -e "  task:"
    echo -e "    execution:"
    echo -e "      pool:"
    echo -e "        core-size: ${core_threads}"
    echo -e "        max-size: ${max_threads}"
    echo -e "        queue-capacity: ${queue_size}"
    echo -e "        keep-alive: 60s"
    
    # 添加Java代码形式的线程池配置示例
    echo -e "\n# Java代码配置"
    echo -e "@Bean"
    echo -e "public ThreadPoolTaskExecutor taskExecutor() {"
    echo -e "    ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();"
    echo -e "    executor.setCorePoolSize(${core_threads});"
    echo -e "    executor.setMaxPoolSize(${max_threads});"
    echo -e "    executor.setQueueCapacity(${queue_size});"
    echo -e "    executor.setKeepAliveSeconds(60);"
    echo -e "    executor.setThreadNamePrefix(\"task-\");"
    echo -e "    executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());"
    echo -e "    return executor;"
    echo -e "}"
    
    # 添加Spring项目系统配置推荐
    echo $SEPARATOR
    echo -e "====== Spring项目系统配置推荐 ======"
    echo -e "# JVM参数推荐配置"
    
    # 根据系统内存计算合适的堆内存设置
    local max_heap=$((mem_total_gb * 75 / 100)) # 系统内存75%
    
    # 根据系统内存调整设置
    if [ $max_heap -lt 2 ]; then
        max_heap=2
    elif [ $max_heap -gt 32 ]; then
        max_heap=32  # 一般不建议超过32GB
    fi
    
    local young_ratio=30  # 年轻代比例，默认30%
    
    echo -e "# 推荐JVM参数(${mem_total_gb}GB系统内存):"
    echo -e "JAVA_OPTS=\"-Xms${max_heap}g -Xmx${max_heap}g -XX:NewRatio=${young_ratio} -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+UseStringDeduplication -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp/\""
    
    echo -e "\n# Tomcat连接池配置 (server.tomcat.*)"
    echo -e "server:"
    echo -e "  tomcat:"
    echo -e "    max-threads: $((cpu_cores * 20))"
    echo -e "    min-spare-threads: $((cpu_cores * 5))"
    echo -e "    max-connections: $((cpu_cores * 100))"
    echo -e "    accept-count: $((cpu_cores * 30))"
    echo -e "    connection-timeout: 20000"
    
    echo -e "\n# 数据库连接池配置 (spring.datasource.hikari.*)"
    echo -e "spring:"
    echo -e "  datasource:"
    echo -e "    hikari:"
    echo -e "      maximum-pool-size: $((cpu_cores * 5 < 20 ? cpu_cores * 5 : 20))"
    echo -e "      minimum-idle: $((cpu_cores < 5 ? cpu_cores : 5))"
    echo -e "      idle-timeout: 600000"
    echo -e "      connection-timeout: 30000"
    echo -e "      max-lifetime: 1800000"
    
    echo -e "\n# Redis连接池配置"
    echo -e "spring:"
    echo -e "  redis:"
    echo -e "    lettuce:"
    echo -e "      pool:"
    echo -e "        max-active: $((cpu_cores * 4 < 32 ? cpu_cores * 4 : 32))"
    echo -e "        max-idle: $((cpu_cores * 2 < 16 ? cpu_cores * 2 : 16))"
    echo -e "        min-idle: $((cpu_cores < 8 ? cpu_cores : 8))"
    echo -e "        max-wait: 1000ms"
    echo -e "    timeout: 5000ms"
    
    # 根据当前系统负载提供建议
    if (( $(echo "$load_per_core > 0.7" | bc -l) )); then
        echo -e "\n${YELLOW}注意：当前系统负载较高，建议适当减小以上配置参数值${NC}"
    elif (( $(echo "$mem_usage > 80" | bc -l) )); then
        echo -e "\n${YELLOW}注意：当前系统内存使用率较高，建议减小堆内存大小${NC}"
    fi
    
    echo $SEPARATOR
    read -p "按回车键返回主菜单..."
}


# 查看多节点Tomcat日志
view_all_nodes_logs() {
    clear
    echo "====== 多节点Tomcat日志统一查看 ======"
    echo "当前集群节点: ${!NODES[@]}"
    echo $SEPARATOR
    
    # 检查系统中是否安装了multitail工具
    local use_multitail=false
    if command -v multitail >/dev/null 2>&1; then
        use_multitail=true
    fi
    
    echo "1: 实时查看所有节点的最新日志"
    echo "2: 按关键词过滤所有节点的日志"
    echo "3: 只查看错误和异常日志"
    echo "4: 查看特定节点组合的日志"
    echo "b: 返回主菜单"
    echo $SEPARATOR
    
    read -e -p "请选择查看方式: " log_option
    case $log_option in
        1)
            clear
            echo "正在统一查看所有节点的最新日志 (Ctrl+C 退出)..."
            echo $SEPARATOR
            
            # 准备所有节点的日志文件路径
            local log_files=""
            for node in "${!NODES[@]}"; do
                local tomcat_path=$(get_tomcat_node_path "$node")
                log_files="$log_files $tomcat_path/logs/catalina-daemon.out"
            done
            
            if $use_multitail; then
                # 使用multitail工具进行更友好的显示
                multitail -s 2 --mergeall -ci red -e "ERROR|SEVERE|Exception" $log_files
            else
                # 使用tail命令的替代方案
                tail -f $log_files | awk '{print strftime("%Y-%m-%d %H:%M:%S"), $0}'
            fi
            ;;
        2)
            read -e -p "请输入要过滤的关键词: " keyword
            if [ -z "$keyword" ]; then
                echo "关键词不能为空"
                sleep 2
                view_all_nodes_logs  # 重新调用函数
                return
            fi
            
            clear
            echo "正在过滤所有节点日志中包含 \"$keyword\" 的内容 (Ctrl+C 退出)..."
            echo $SEPARATOR
            
            # 准备grep命令
            local grep_cmd="grep -i \"$keyword\""
            local log_files=""
            for node in "${!NODES[@]}"; do
                local tomcat_path=$(get_tomcat_node_path "$node")
                local log_file="$tomcat_path/logs/catalina-daemon.out"
                
                if $use_multitail; then
                    # 收集文件路径，稍后一次性传递给multitail
                    log_files="$log_files $log_file"
                else
                    # 为每个节点创建单独的tail进程
                    echo "正在监控 [$node] 日志..."
                    tail -f "$log_file" | grep -i --line-buffered "$keyword" | sed "s/^/[$node] /" &
                    tail_pids="$tail_pids $!"
                fi
            done
            
            if $use_multitail; then
                # 使用multitail进行彩色高亮显示
                multitail -s 2 --mergeall -cS tomcat -I "$keyword" $log_files
            else
                # 等待用户中断(Ctrl+C)
                echo "正在监控所有节点日志，按Ctrl+C退出..."
                wait
                
                # 清理所有tail进程
                for pid in $tail_pids; do
                    kill $pid 2>/dev/null
                done
            fi
            ;;
        3)
            clear
            echo "正在监控所有节点的错误和异常日志 (Ctrl+C 退出)..."
            echo $SEPARATOR
            
            local log_files=""
            for node in "${!NODES[@]}"; do
                local tomcat_path=$(get_tomcat_node_path "$node")
                local log_file="$tomcat_path/logs/catalina-daemon.out"
                
                if $use_multitail; then
                    log_files="$log_files $log_file"
                else
                    echo "正在监控 [$node] 错误日志..."
                    tail -f "$log_file" | grep -i --line-buffered "error\|exception\|severe" | sed "s/^/[$node] /" &
                    tail_pids="$tail_pids $!"
                fi
            done
            
            if $use_multitail; then
                # 使用multitail并高亮显示错误和异常
                multitail -s 2 --mergeall -ci red -e "ERROR|SEVERE|Exception|WARN" $log_files
            else
                # 等待用户中断(Ctrl+C)
                echo "正在监控所有节点错误日志，按Ctrl+C退出..."
                wait
                
                # 清理所有tail进程
                for pid in $tail_pids; do
                    kill $pid 2>/dev/null
                done
            fi
            ;;
        4)
            # 让用户选择要查看的节点
            echo "请选择要监控的节点 (输入节点编号，用空格分隔):"
            local i=1
            for node in "${!NODES[@]}"; do
                echo "$i: $node"
                i=$((i+1))
            done
            
            read -e -p "请输入节点编号: " selected_nodes
            if [ -z "$selected_nodes" ]; then
                echo "未选择任何节点"
                sleep 2
                view_all_nodes_logs  # 重新调用函数
                return
            fi
            
            clear
            echo "正在监控选定节点的日志 (Ctrl+C 退出)..."
            echo $SEPARATOR
            
            # 处理用户输入的节点编号
            local node_array=(${!NODES[@]})
            local log_files=""
            local tail_pids=""
            
            for num in $selected_nodes; do
                if [ "$num" -gt 0 ] && [ "$num" -le "${#NODES[@]}" ]; then
                    local idx=$((num - 1))
                    local node="${node_array[$idx]}"
                    echo "监控节点: $node"
                    
                    local tomcat_path=$(get_tomcat_node_path "$node")
                    local log_file="$tomcat_path/logs/catalina-daemon.out"
                    
                    if $use_multitail; then
                        log_files="$log_files $log_file"
                    else
                        tail -f "$log_file" | sed "s/^/[$node] /" &
                        tail_pids="$tail_pids $!"
                    fi
                fi
            done
            
            if $use_multitail; then
                multitail -s 2 --mergeall $log_files
            else
                # 等待用户中断(Ctrl+C)
                echo "正在监控选定节点日志，按Ctrl+C退出..."
                wait
                
                # 清理所有tail进程
                for pid in $tail_pids; do
                    kill $pid 2>/dev/null
                done
            fi
            ;;
        b|B)
            return
            ;;
        *)
            echo "无效选项"
            sleep 1
            view_all_nodes_logs  # 重新调用函数
            ;;
    esac
    
    # 清理可能残留的进程
    if [ -n "$tail_pids" ]; then
        for pid in $tail_pids; do
            kill $pid 2>/dev/null || true
        done
    fi
}

# 修改主程序入口
main() {        
    check_prerequisites
    while true; do
        show_menu
        read -e -p "请输入选项: " choice
        case $choice in
            1|2|3)
                tomcat_node_manager_menu $choice
                ;;
            r|R)
                ;;
            "backup"|"BACKUP")
                backup_project
                ;;
            "backup lib"|"BACKUP LIB")
                backup_lib
                ;;
            "logs"|"LOGS")
                view_all_nodes_logs
                ;;
            "du -sh backup")
                show_backup_size
                ;;
            "clean backup")
                clean_backup
                ;;
            "du -sh tomcat")
                show_tomcat_size
                ;;
            "mq")
                manage_activemq
                ;;
            "nginx")
                manage_nginx
                ;;
            "restart")
                restart_all_nodes
                ;;
            "start")
                start_all_nodes
                ;;
            "stop")
                stop_all_nodes
                ;;
            "system")
                evaluate_system_health
                ;;
            q|Q)
                echo "正在退出程序..."
                exit 0
                ;;
            *)
                echo "无效选项"
                sleep 1
                ;;
        esac
    done
}

# 启动主程序
main